﻿using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using log4net;
using Snap.Model;
using Snap.RssManager;

namespace News
{
    public class NewsUpdater
    {
        //public static readonly string NYSE = "NYSE";
        //public static readonly string NASDAQ = "NASDAQ";
        //public static readonly string AMEX = "AMEX";

        public static readonly List<string> ApprovedSources = new List<string> { "reuters" };

        private static readonly Stopwatch StopWatch = new Stopwatch();
        private static readonly ILog Logger =
            LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);


        /// <summary>
        /// Pulls the RSS feeds.
        /// </summary>
        public static void PullRssFeeds()
        {
            // Get the basic URL structure for our RSS feeds.
            var rssUrlBase = Properties.Settings.Default.RssUrlBase;

            // Our final structure looks like this:
            // http://www.google.com/finance/company_news?q=[Exchange]:[Symbol]&output=rss
            // alternative1: http://www.reuters.com/finance/stocks/companyNews?symbol=AA&output=rss
            Logger.Debug("Get all securities in the Russell 1000");
            var contextForMapping = new SnapModelContainer();
            var symbolToExchangeMap = contextForMapping.Companies
                .Where(c => c.IsRussell1000)
                .ToDictionary(c => c.Symbol, c => c.Exchange); // We need exchange for RSS query

            Logger.DebugFormat("Retrieved {0} symbols.", symbolToExchangeMap.Count);

            Logger.Debug("Create and populate map for relating news items to symbols");
            Logger.Debug("  <symbol, <key (title+date), newsItem>>");

            Logger.Debug("First, get Russell symbols that have news items already");
            var symbolsThatHaveNewsItems = contextForMapping.NewsToCompanies
                .Select(ntc => ntc.CompanySymbol)
                .Distinct().ToList();
            symbolsThatHaveNewsItems = symbolsThatHaveNewsItems
                .Where(symbolToExchangeMap.ContainsKey)
                .ToList();
            Logger.Debug("Then, convert this to a Dictionary<string, Dictionary<string, string>>");
            var masterMap = new Dictionary<string, Dictionary<string, string>>();

            foreach (var symbol in symbolsThatHaveNewsItems)
            {
                string currentSymbol = symbol;
                // Key for inner dictionary is Title+PublicationDate. Value is NewsItem
                var newsItemsForSymbol = (from ntc in contextForMapping.NewsToCompanies
                                          join newsItem in contextForMapping.NewsItems
                                              on ntc.NewsItemId equals newsItem.Id
                                          where ntc.CompanySymbol == currentSymbol
                                          select newsItem).ToDictionary(
                                              item => String.Concat( item.Title, item.PublicationDate.Date.ToString() ),
                                              item => String.Empty );
                masterMap[symbol] = newsItemsForSymbol;
            }

            var existingNewsItems =
                contextForMapping.NewsItems.ToDictionary( n => String.Concat( n.Title, n.PublicationDate.Date ), n => n );
            var newSymbolToNewsMap = new Dictionary<string, Dictionary<string, NewsItem>>();
            Logger.Debug("Capture metrics for pulling news items and storing them.");
            StopWatch.Restart();
            int successfulPulls = 0;
            int failedPulls = 0;
            foreach (var symbol in symbolToExchangeMap.Keys)
            {
                var context = new SnapModelContainer();
                string rssUrl = rssUrlBase + symbolToExchangeMap[symbol] + ":" + symbol + "&output=rss";
                var rssManager = new Manager(rssUrl);
                Collection<Rss.Item> rssResults;
                try
                {
                    rssResults = rssManager.GetFeed();
                }
                catch (WebException we)
                {
                    Logger.ErrorFormat("Caught a WebException trying to create RSS object for URL [{0}]", rssUrl);
                    Logger.Error("Exception text: ", we);
                    rssResults = new Collection<Rss.Item>();
                }

                // Regex to parse source from link
                var regex = new Regex(@"(?<=[http://www.|http://])\w+(?=.com)");
                foreach (var rssItem in rssResults)
                {
                    var match = regex.Match(rssItem.Link);
                    string source = match.Success ? match.Value : string.Empty;
                    // Do not keep any news from non-approved sources
                    if (!ApprovedSources.Contains(source)) continue;

                    var newsItem = new NewsItem
                    {
                        Body = string.Empty,
                        Description = rssItem.Description,
                        Title = Regex.Replace(rssItem.Title, @"<(.|\n)*?>", String.Empty),
                        PublicationDate = rssItem.Date.ToUniversalTime(),
                        Link = rssItem.Link,
                        Source = source
                    };

                    // We are only interested in news items that occur during market hours (M-F, 0700 - 2000 EST [0800 - 2100 CST])
                    // So, knowing we store Publication Dates as UTC, we can use TimeZone.Current to convert to local and
                    //  test against that time (UTC will span date boundry).
                    if (IsNotValidTime(TimeZone.CurrentTimeZone.ToLocalTime(newsItem.PublicationDate))) continue;

                    Logger.DebugFormat("Pulling the news body for \"{0}\"", newsItem.Title);
                    try
                    {
                        string rawBody = new WebClient().DownloadString(newsItem.Link);
                        //newsItem.Body = Regex.Replace( rawBody, @"<(.|\n)*?>", String.Empty );
                        // For now, use raw Body. We can parse it later.
                        newsItem.Body = rawBody;
                    }
                    catch (Exception ex)
                    {
                        // Don't keep items without a body
                        Logger.ErrorFormat("Caught Exception trying to pull news body from [{0}]", newsItem.Link);
                        Logger.Error("Exception text: ", ex);
                        failedPulls++;
                        continue;
                    }

                    string key = String.Concat( newsItem.Title, newsItem.PublicationDate.Date );
                    // If we've seen this news item for this symbol before, drop everything and continue.
                    if (masterMap.ContainsKey(symbol) && masterMap[symbol].ContainsKey(key)) continue;

                    // Update SymbolToNewsMaps if we haven't seen this relation before
                    if (!newSymbolToNewsMap.ContainsKey(symbol))
                    {
                        newSymbolToNewsMap[symbol] = new Dictionary<string, NewsItem>();
                    }
                    if (!existingNewsItems.ContainsKey(key))
                    {
                        existingNewsItems[key] = newsItem;
                        Logger.DebugFormat("Storing NewsItem with key of: {0}", key);
                        context.NewsItems.AddObject(newsItem);
                    }
                    
                    // If we've never seen this relation before, store the symbol-news relationship for later use.
                    newSymbolToNewsMap[symbol][key] = newsItem;
                    successfulPulls++;
                }

                // Save the news items
                context.SaveChanges();
            }
            StopWatch.Stop();
            Logger.DebugFormat("Pulled and saved [{0}] news items while Failing [{1}] times.", successfulPulls, failedPulls);
            Logger.DebugFormat("Process took {0} and averaged {1} news item per second",
                                StopWatch.Elapsed, (successfulPulls / (StopWatch.ElapsedMilliseconds / 1000)));

            // Take the symbol to news map and store it
            var symbolToNewsContext = new SnapModelContainer();
            var allExistingNewsItems =
               symbolToNewsContext.NewsItems.ToDictionary(n => (String.Concat(n.Title, n.PublicationDate.Date.ToString())),
                                                         n => n.Id);
            // Only store the first item of the day, in case same article is broadcast more than once.
            foreach (var newsItem in symbolToNewsContext.NewsItems)
            {
                var key = String.Concat(newsItem.Title, newsItem.PublicationDate.Date.ToString());
                if (!allExistingNewsItems.ContainsKey(key))
                {
                    allExistingNewsItems[key] = newsItem.Id;
                }
            }
            //existingNewsItems = symbolToNewsContext.NewsItems.ToDictionary(
            //    n => (String.Concat( n.Title, n.PublicationDate.Date.ToString() )),
            //    n => n.Id );
            StopWatch.Restart();
            foreach (var symbol in newSymbolToNewsMap.Keys)
            {
                foreach (var keyToNewsItem in newSymbolToNewsMap[symbol].Values)
                {
                    string key = String.Concat(keyToNewsItem.Title, keyToNewsItem.PublicationDate.Date.ToString());
                    int newsId = allExistingNewsItems[key];
                    symbolToNewsContext.NewsToCompanies.AddObject(new NewsToCompany
                    {
                        CompanySymbol = symbol,
                        NewsItemId = newsId
                    });
                }
            }
            symbolToNewsContext.SaveChanges();
            int numberOfMappingsSaved = newSymbolToNewsMap.Values.Sum(d => d.Values.Count);
            Logger.DebugFormat("Stored [{0}] SymbolToNews mappings in [{1}] ({2} per second)",
                                numberOfMappingsSaved,
                                StopWatch.Elapsed,
                                (numberOfMappingsSaved / (StopWatch.ElapsedMilliseconds / 1000.0))
                );

        }

        private static bool IsNotValidTime( DateTime cdt )
        {
            if (cdt.DayOfWeek == DayOfWeek.Saturday || cdt.DayOfWeek == DayOfWeek.Saturday)
            {
                Logger.DebugFormat("News event occurred on {0}. Cannot process news on non-trading days", cdt.DayOfWeek);
                return true;
            }
            var marketOpenCDT = new DateTime(cdt.Year, cdt.Month, cdt.Day, 8, 0, 0);
            var marketCloseCDT = new DateTime(cdt.Year, cdt.Month, cdt.Day, 21, 0, 0);
            if (cdt < marketOpenCDT && cdt >= marketCloseCDT)
            {
                Logger.DebugFormat("News event occurred outside market hours: {0:HH:mm:ss}", cdt);
                return true;
            }

            return false;
        }
    }
}
